package cc.togetherit.jcrawler.fetch;

import java.net.MalformedURLException;
import java.net.URL;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import cc.togetherit.jcrawler.checkRepeat.RepeatCheck;
import cc.togetherit.jcrawler.dataPool.CrawlerPool;

/****
 * XPath FetchEngine
 * @author jackie
 *
 */
public class JsoupFetchEngine extends AbstractFetchEngine {

    
    
    public JsoupFetchEngine(CrawlerPool<String> downPool, CrawlerPool<FetchRequest> fetchPool, CrawlerPool<FetchRequest> parsePool,RepeatCheck check) {
        super(downPool, fetchPool, parsePool,check);
    }

    
    @Override
    public boolean visited(String link) {
        return this.getCheck().visited(link);
    }

    public FetchResponse fetch(FetchRequest content) {
        Document doc = Jsoup.parse(content.getContent());
        FetchResponse response = new FetchResponse();
        if(doc==null){
            return  response;
        }
        URL u;
        try {
            
            u = new URL(content.getUrl());
            Elements els = doc.select("a");
            String link = null;
            for(Element e:els){
                e.attr("href");
                if (link != null && !link.startsWith("javascript")) { // remove
//                    if(!visited(link) && shouldFetched(link)){
                        response.addLink(new URL(u,link).toString());
//                    }
                }
            }
        } catch (MalformedURLException e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        }
       
        return response;
    }


    public boolean shouldFetched(String url) {
        // TODO Auto-generated method stub
        return false;
    }


    public boolean shouldParsed(String url) {
        // TODO Auto-generated method stub
        return false;
    }

  

}
